#coding=utf-8

import urllib2
from pyquery import PyQuery
import logging
import codecs
import time

host="http://www.530p.com"
addr="http://www.530p.com/lishi/daweigongting-181063/20771488.htm"

def clearFile():
    try:
        file=codecs.open("dwgt.txt",'a',encoding='utf-8')
        file.truncate()
        file.close()
        logging.warning("----clear file----")
        return True
    except BaseException:
        logging.warning("----clear file error----")
        return False

def writeFile(data):
    try:
        file = codecs.open("dwgt.txt", 'a',encoding='utf-8')
        file.writelines(data+"\n")
        file.close()
        return True
    except BaseException ,e:
        logging.warning("----open file error----:")
        logging.exception(e)
        return False

def scrape(addr):
    try:
        req = urllib2.Request(addr)
        html = urllib2.urlopen(req,timeout=5).read().decode('gbk','ignore').encode('utf-8')
        html=html.replace("gbk","utf-8")
        query=PyQuery(html)
        title=query("#cps_title").text()
        content=query("#cp_content").text()
        if title.__len__()==0:
            return False,"end"
        next=query("#nextLink").attr("href")
        writeFile(title)
        writeFile(content)
        return True,str(next)
    except BaseException,e:
        logging.exception(e)
        return False,None

if __name__=="__main__":
    hasNext=True
    i=1
    try:
        clearFile()
        while hasNext==True:
            print addr
            (hasNext, link) = scrape(addr)
            #print type(link)

            if link is None:
                print "----connect error, repeate----"
                hasNext=True
                continue
            if link=="end":
                print "----execute complete----"
                break
            logging.warning(link)
            addr=host+link
            logging.warning("----Chapter:%d is complete----",i)
            i+=1
            time.sleep(0)
    except BaseException,e:
        logging.exception(e)

